# import requests
import json
import asyncio
import aiohttp


from pymongo import MongoClient
from gridfs import GridFSBucket

db = MongoClient('localhost:28001')['test']
fs = GridFSBucket(db,bucket_name='fs', chunk_size_bytes=1024,)

async def get_page(work_urls, name):
    conn = aiohttp.TCPConnector(limit=10)
    async with aiohttp.ClientSession(connector=conn) as session:
        for _ in range(4):
            try:
                with aiohttp.Timeout(20):
                    async with session.get(work_urls) as r:
                        assert r.status == 200
                        chunk  = await r.read()
                        # fd.write(chunk)
                        fs.upload_from_stream(
                            "test.jpg",
                            chunk,
                            metadata={"contentType": "image/jpeg"})
                        break
            except Exception as e:
                print(e)


def main():

    
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    tasks = []
    with open('niupic.json', 'rt', encoding='utf8')as f:
        for j,line in enumerate(f):
            dic = json.loads(line)
            pics = dic.get('result').get('pics')

            for i,pic in enumerate(pics):
                tasks.append(asyncio.ensure_future(get_page(pic,'%s_%s'%(j,i))))
                loop.run_until_complete(asyncio.wait(tasks))
            break

    
    loop.close()

if __name__ == '__main__':
    main()
# def get_pic(url,name):
#     req = requests.get(url, timeout=40)

#     with open('%s.jpg'%name,'wb')as f:
#       f.write(req.content)

# 
